import pandas as pd
import numpy as np
from environment import Env
from RL import QLearningTable
import matplotlib.pyplot as plt

def update(iteration_num , plane_num):
    t = []
    tn = 0
    for episode in range(iteration_num):
        plane_list = []
        observation , nowTime = env.reset()   # 初始化 state 的观测值
        while True:
            # 根据现在时间，拿到状态与环境
            observation , state = env.update_env(plane_num , nowTime)

            # RL 大脑根据 state 的观测值挑选 action
            action = RL.choose_action(observation)
            #print(action, observation, state)
            # 探索者在环境中实施这个 action, 并得到环境返回的下一个 state 观测值, reward 和 done (是否是掉下地狱或者升上天堂)
            observation_, reward, state_ , nowTime_ , first_plane = env.step(observation , action , state , nowTime)

            plane_list.append(first_plane)

            # RL 从这个序列 (state, action, reward, state_) 中学习
            RL.learn(observation, action, reward, observation_ )
            # 将下一个 state与 observation 的值传到下一次循环
            observation = observation_
            state = state_
            nowTime = nowTime_

            # 如果掉下地狱或者升上天堂, 这回合就结束了
            if state == []:
                break
        dtime = env.carculate_dtime(plane_list)
        print(plane_list , dtime)
        t.append(dtime)
    return t
        #print(env.Timetable)


if __name__ == "__main__":
    iteration_num = 400
    plane_num = 10
    env = Env()

    RL = QLearningTable(actions=list(range(env.n_actions)))
    line_fcfs_record = env.calculate_fcfs_dtime(plane_num , iteration_num)
    line_rl_record = update(iteration_num , plane_num)


    # 绘制图像
    plt.plot(line_rl_record, label="RL", marker="o", color="blue", linestyle="-")
    plt.plot(line_fcfs_record, label="FCFS", marker="o", color="red", linestyle="-")
    plt.ylabel("Delay Time")
    plt.xlabel("Number of Iteration")
    plt.legend()
    plt.show()